class UrlManager:
    '''
    url管理器
    '''
    def __init__(self):
        self.new_url = set()    #未爬取的url
        self.old_url = set()    #已爬取的url

    #未爬取集合大小
    def new_url_size(self):
        return len(self.new_url)

    #已爬取集合大小
    def old_url_size(self):
        return len(self.old_url)

    #是否有未爬取的url
    def has_new_url(self):
        return self.new_url_size != 0

    #获取未爬取的url
    def get_new_url(self):
        new_url = self.new_url.pop()
        self.old_url.add(new_url)
        return new_url

    #将新的url放入未爬取集合
    def add_new_url(self,url):
        if url is None:
            return
        if url not in self.new_url and url not in self.old_url:
            self.new_url.add(url)

    #将新的urls放入未爬取集合
    def add_new_urls(self,urls):
        if len(urls) == 0:
            return
        for url in urls:
            self.new_url.add(url)
